use strict;
use warnings;

package Lingua::EN::WSD::WebBased::Vectorizer;

use Lingua::EN::WSD::WebBased::Logger;
use Lingua::EN::WSD::WebBased::Utils;

use Class::AccessorMaker {
    wordnet                     => undef,
    stemmer                     => undef,
    clear_html                  => 1,
    remove_stopwords            => 1,
    retain_only_concept_words   => 1,
    do_stemming                 => 1,
    do_basic_filtering          => 1
}, 'new_init';

sub init {
    
    my $self = shift;

    check_type 'attribute wordnet'
        => $self->wordnet => 'WordNet::QueryData';
    
    use stem;
    $self->{stemmer} = new stem($self->wordnet);

    return $self;

}

# convert a string into a tf vector

sub vectorize {
    
    my ($self, $str) = @_;

    use Lingua::EN::Splitter qw/words/;
    use Lingua::EN::StopWords qw/%StopWords/;

    $str = clear_html($str) if $self->clear_html;
    
    my @words = map { lc } @{words $str};


    @words = grep { /\w\w/ && /^[a-z\-]+$/ } @words
        if $self->do_basic_filtering;
    @words = grep { !$StopWords{$_} } @words
        if $self->remove_stopwords;
    @words = map { $self->stemmer->stemWord($_) } @words
        if $self->do_stemming;
    @words = grep { $self->wordnet->queryWord($_) > 0 } @words
        if $self->retain_only_concept_words;
    @words = grep { /\w\w/ && /^[a-z\-]+$/ } @words
        if $self->do_basic_filtering;
    
    my %vector;
    $vector{$_}++ for @words;

    return \%vector;

}

1;
